#! /usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals, absolute_import, division, print_function, absolute_import


from bs4 import BeautifulSoup
from bs4 import Tag
from collections import OrderedDict
from datetime import datetime
from importlib import import_module
from lxml import etree
from requests.models import Response
from time import sleep
import click
import copy
import glob
import json
import logging
import math
import os
import random
import re
import requests
import signal
import sys
import time
import urllib.request
import os
import shutil
import traceback
import difflib



def correct_json():
    words_dict = {}

    f = open("ciba.com_20200226.json", 'r', -1, encoding="utf_8_sig")
    data = f.read()
    f.close()

    d = json.loads(data)

    for word, entry in d.items():

        words_dict[word] = {}

        for entry_key, entry_v in entry.items():
            if entry_key == "forms":
                forms = entry_v
                n_forms = {}
                for fk, fv in forms.items():
                    n_fv = []
                    if type(fv) == str:
                        fv = [fv]
                    for fvv in fv:
                        n_fv.append(fvv.strip())
                    n_forms[fk.strip()] = n_fv

                words_dict[word]["forms"] = n_forms

            else:
                words_dict[word]["definations"] = []

                # 其他为pos
                pos = entry_key
                pos_list = None
                if pos != "unkown":
                    pos = pos.replace("modeal verb", "modal verb")
                    pos_list = pos.split(",")

                for pos_key, pos_v in entry_v.items():
                    if pos_key != "definations":
                        print("pos_key " + pos_key)
                        continue

                    defs = pos_v

                    for def_key, def_list in defs.items():
                        if def_key != "from iciba.com, 金山词霸":
                            print("def_key " + def_key)
                            continue

                        for def_ in def_list:
                            if pos_list:
                                def_["pos"] = pos_list
                            words_dict[word]["definations"].append(def_)
                            # 添加pos信息
                            pass



    f_processed = open("ciba.com_20200226.json_", 'w', -1, encoding="utf_8_sig")
    f_processed.write(json.dumps(words_dict, indent=4, sort_keys=True, ensure_ascii=False))
    f_processed.close()

def parse_all_in_one_css():
    f = open("wordnik_20200221.json", 'r', -1, encoding="utf_8_sig")
    data = f.read()
    f.close()

    f_debug = open("1.debug.txt", 'w', -1, encoding="utf_8_sig")

    def dump(o, path):
        if type(o) is dict:
            for k, v in o.items():
                dump(v, path + " -> " + k)
        elif type(o) is list:
            for i in o:
                dump(i, path + " [ ")
        else:
            f_debug.write(path+"\n")

    d = json.loads(data)
    for k, v in d.items():
        dump(v, "")

    f_debug.close()

#parse_all_in_one_css()

def parse_correct_wordnik():
    f = open("wordnik_20200221.json", 'r', -1, encoding="utf_8_sig")
    data = f.read()
    f.close()

    f_debug = open("1.debug.txt", 'w', -1, encoding="utf_8_sig")

    d = json.loads(data)

    word_dict = {}

    for word, word_info_dict in d.items():
        extra = word_info_dict.get("extra")
        extra_forms = None
        extra_related = None
        if extra:
            extra_forms = extra.get("forms")
            extra_related = extra.get("related")

        for pos, pos_dict in word_info_dict.items():
            if pos == "extra":
                continue
            else:
                definations = pos_dict.get("definations")
                if definations:
                    for source, source_dict in definations.items():
                        for def_dict in source_dict:
                            defination = def_dict.get("defination")
                            extra = def_dict.get("extra")

                            if word_dict.get(source) is None:
                                word_dict[source] = {}

                            cur_word_dict = word_dict[source]

                            if cur_word_dict.get(word) is None:
                                cur_word_dict[word] = {}

                            if extra_forms:
                                cur_word_dict[word]["related_forms"] = extra_forms
                            if extra_related:
                                cur_word_dict[word]["related_words"] = extra_related

                            if cur_word_dict[word].get("definations") is None:
                                cur_word_dict[word]["definations"] = []

                            def_item = {}
                            if defination:
                                def_item["defination"] = defination
                            if extra:
                                def_item["extra"] = extra
                            if pos != "unknow":
                                def_item["pos"] = pos

                            cur_word_dict[word]["definations"].append(def_item)

    f_debug.close()

    for source, di in word_dict.items():
        f_debug = open(source.replace("/","")+".wordnik.json", 'w', -1, encoding="utf_8_sig")
        f_debug.write(json.dumps(di, indent=4, sort_keys=True, ensure_ascii=False))
        f_debug.close()


def parse_bing():
    f_debug = open("1.debug.txt", 'w', -1, encoding="utf_8_sig")
    count = 0
    f = open("bing.csv", 'r', -1, encoding="utf_8_sig")
    last_item = None
    for line in f:
        count += 1
        cols  = line.split("|||")
        is_good = True
        if cols[0].strip() == "defination:":
            #f_debug.write("pos: "+cols[2]+"\n")
            pass

        elif cols[0].strip() == "shape:":
            #f_debug.write("shape: "+cols[2]+"\n")
            pass

        elif cols[0].strip() == "meaning:":
            #f_debug.write("meaning: "+cols[2]+" " +cols[3]+"\n")
            pass

        elif cols[0].strip() == "antoid:":
            #f_debug.write("antoid: "+cols[2]+"\n")
            pass

        elif cols[0].strip() == "synoid:":
            #f_debug.write("synoid: "+cols[2]+"\n")
            pass

        elif cols[0].strip() == "collation:":
            #f_debug.write("collation: "+cols[2]+"\n")
            pass

        elif cols[0].strip() == "idom:":
            # idom: ||| abreast||| keep abreast of sth|||了解最新情况；跟上（某事物的发展） to make sure that you know all the most recent facts about a subject It is almost impossible to keep abreast of all the latest developments in computing. 要跟上计算机领域所有最新的发展几乎不可能。
            #f_debug.write("idom: "+"\n")
            pass
        else:
            is_good = False

        if is_good:
            if last_item: f_debug.write(last_item.strip()+"\n")
            last_item = line
        else:
            last_item += line
            last_item = re.sub("[ \t\n\r]+", " ",last_item)
            #f_debug.write("?? "+str(count)+" "+cols[0]+"\n")
    if last_item: f_debug.write(last_item.strip()+"\n")
    f.close()
    f_debug.close()

pos_list = {
    "abbr."            : "abbr."            ,
    "adj."             : "adj."             ,
    "adjs."            : "adj."            , #adj
    "adv."             : "adv."             ,
    "art."             : "art."             ,
    "auxv."            : "aux."            ,
    "conj."            : "conj."            ,
    "defa."            : "art."            , #the , 限定性冠词
    "det."             : "det."             ,
#    "infinmarker."     : "infinmarker."     , # to 不定式标识符
    "int."             : "int."             ,
    "linkv."           : "link verb"           ,
    "modalv."          : "modal verb"          ,
    "modv."            : "modal verb"            ,
    "n."               : "n."               ,
    "na."              : "n."              ,
    "nb."              : "n."              ,
    "np."              : "n."              ,
    "ns."              : "n."              ,
    "num."             : "num."             ,
    "ordnumber."       : "ord."       ,
#    "pf."              : "pf."              ,
#    "ph."              : "ph."              ,
    "pref."            : "pref."            , # 前缀
    "prep."            : "prep."            ,
    "pron."            : "pron."            ,
    "sf."              : "short."              , # 缩写
    "short."           : "short."           , # 缩写
    "suf."             : "suff."             , # 后缀
    "suff."            : "suff."            ,
    "sym."             : "sym."             ,
    "symb."            : "symb."            ,
    "un."              : "n."              ,
    "v."               : "v."               ,
    "vt."              : "vt."              ,
    "vi."              : "vi."              ,
}

def correct_bing():
    f_debug = open("1.debug.txt", 'w', -1, encoding="utf_8_sig")
    count = 0
    f = open("bing.csv", 'r', -1, encoding="utf_8_sig")

    word_dict = {}
    for line in f:
        count += 1
        cols  = line.split("|||")
        is_good = True
        if cols[0].strip() == "defination:":
            #f_debug.write("pos: "+cols[2]+"\n")
            word = cols[1].strip()
            pos = cols[2].strip()
            pos_cn = cols[3].strip()
            def_ = cols[4].strip()

            if word_dict.get(word) is None:
                word_dict[word] = {}

            if word_dict[word].get("definations") is None:
                word_dict[word]["definations"] = []

            def_item = {}
            def_item["defination"] = def_


            if pos != "pos,web":
                if  pos_list.get(pos_cn.strip()):
                    def_item["pos"] = pos_list.get(pos_cn.strip())  
                else:
                    f_debug.write(line+"\n")
            else:
                def_item["source"] = pos_cn

            dup = False
            for d1 in word_dict[word]["definations"]:
                if str(d1) == str(def_item):
                    dup = True
                    break
            if dup is False:
                word_dict[word]["definations"].append(def_item)

            pass

        elif cols[0].strip() == "shape:":
            word = cols[1].strip()
            shape = cols[2].strip()
            word2 = cols[3].strip()

            forms = {
                "过去式："      : "past tense",
                "过去分词："    : "past participle",
                "第三人称单数：": "third person singular",
                "现在分词："    : "present participle",
                "比较级："      : "comparative",
                "最高级："      : "superlative",
                "复数："        : "plural",
                "单数："        : "singular",
            }

            if forms.get(shape.strip()):
                shape = forms[shape.strip()]
            else:
                continue

            if word_dict.get(word) is None:
                word_dict[word] = {}

            if word_dict[word].get("forms") is None:
                word_dict[word]["forms"] = {}

            if word_dict[word]["forms"].get(shape) is None:
                word_dict[word]["forms"][shape]= []

            if word2 not in word_dict[word]["forms"][shape]:
                word_dict[word]["forms"][shape].append(word2)

            #f_debug.write(shape+"\n")
            pass

        elif cols[0].strip() == "meaning:":
            word = cols[1].strip()
            source = cols[2].strip()
            pos = cols[3].strip()
            def_ = cols[4].strip()

            if word_dict.get(word) is None:
                word_dict[word] = {}

            if word_dict[word].get("definations") is None:
                word_dict[word]["definations"] = []

            def_item = {}
            def_item["defination"] = def_

            if pos != "pos,web":
                #def_item["pos"] = pos
                if  pos_list.get(pos.strip()):
                    def_item["pos"] = pos_list.get(pos.strip()) 
                else:
                    f_debug.write(line+"\n")

            def_item["source"] = source

            dup = False
            for d1 in word_dict[word]["definations"]:
                if str(d1) == str(def_item):
                    dup = True
                    break
            if dup is False:
                word_dict[word]["definations"].append(def_item)
            pass

        elif cols[0].strip() == "antoid:":
            word = cols[1].strip()
            pos = cols[2].strip()
            word2 = cols[3].strip()

            if word_dict.get(word) is None:
                word_dict[word] = {}

            if word_dict[word].get("antoid") is None:
                word_dict[word]["antoid"] = []

            nn = pos + " ||| "+ word2
            if  nn not in word_dict[word]["antoid"]:
                word_dict[word]["antoid"].append(nn)
            pass

        elif cols[0].strip() == "synoid:":
            word = cols[1].strip()
            pos = cols[2].strip()
            word2 = cols[3].strip()

            if word_dict.get(word) is None:
                word_dict[word] = {}

            if word_dict[word].get("synoid") is None:
                word_dict[word]["synoid"] = []

            nn = pos + " ||| "+ word2
            if  nn not in word_dict[word]["synoid"]:
                word_dict[word]["synoid"].append(pos + " ||| "+ word2)
            pass

        elif cols[0].strip() == "collation:":
            word = cols[1].strip()
            pos = cols[2].strip()
            word2 = cols[3].strip()

            if word_dict.get(word) is None:
                word_dict[word] = {}

            if word_dict[word].get("collation") is None:
                word_dict[word]["collation"] = []

            nn = pos + " ||| "+ word2
            if  nn not in word_dict[word]["collation"]:
                word_dict[word]["collation"].append(pos + " ||| "+ word2)
            pass

        elif cols[0].strip() == "idom:":
            # idom: ||| abreast||| keep abreast of sth|||了解最新情况；跟上（某事物的发展） to make sure that you know all the most recent facts about a subject It is almost impossible to keep abreast of all the latest developments in computing. 要跟上计算机领域所有最新的发展几乎不可能。
            word = cols[1].strip()
            pos = cols[2].strip()
            if len(cols) == 4:
                word2 = cols[3].strip()
            else:
                word2 = ""

            if word_dict.get(word) is None:
                word_dict[word] = {}

            if word_dict[word].get("idom") is None:
                word_dict[word]["idom"] = []

            nn = pos + " ||| "+ word2
            if  nn not in word_dict[word]["idom"]:
                word_dict[word]["idom"].append(pos + " ||| "+ word2)
            pass
        else:
            f_debug.write("?? "+str(count)+" "+cols[0]+"\n")

    f.close()
    f_debug.close()

    f_processed = open("bing_20200226.json", 'w', -1, encoding="utf_8_sig")
    f_processed.write(json.dumps(word_dict, indent=4, sort_keys=True, ensure_ascii=False))
    f_processed.close()


correct_bing()










